// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
// Copyright (c) 2011, Dr. Masroor Ehsan
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of the author nor the
//       names of its contributors may be used to endorse or promote products
//       derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *

using System;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Text;
using Common.Logging;

namespace dseScraper
{
    internal static class WebDownloader
    {
        private const int DEFAULT_DOWNLOAD_TIMEOUT = 60*1000;

        public static string DownloadHtml(Uri location, string referer, bool enableGzipCompression)
        {
            var log = LogManager.GetLogger("WebDownloader");
            var request = ConfigureRequest(location, referer, enableGzipCompression);
            var result = string.Empty;

            try
            {
                using (var response = (HttpWebResponse) request.GetResponse())
                {
                    var responseStream = response.GetResponseStream();
                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                        responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                        responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);

                    using (var reader = new StreamReader(responseStream, Encoding.Default))
                    {
                        result = reader.ReadToEnd();
                    }
                }
            }
            catch (WebException webError)
            {
                if (webError.Response != null)
                {
                    using (var reader = new StreamReader(webError.Response.GetResponseStream()))
                    {
                        log.Error(reader.ReadToEnd());
                    }
                }
            }
            catch (Exception exc)
            {
                log.Error(exc);
            }
            return result;
        }

        public static string DownloadHtml(Uri location, string referer, bool enableGzipCompression, out DateTime dateTime)
        {
            dateTime = DateTime.UtcNow;
            var log = LogManager.GetLogger("WebDownloader");
            var request = ConfigureRequest(location, referer, enableGzipCompression);
            var result = string.Empty;

            try
            {
                using (var response = (HttpWebResponse)request.GetResponse())
                {
                    if (response.Headers.HasKeys())
                    {
                        foreach (string key in response.Headers.Keys)
                        {
                            if (key.ToUpper().Equals("DATE"))
                            {
                                var value = response.Headers.Get(key);
                                dateTime = DateTime.Parse(value, CultureInfo.InvariantCulture).ToUniversalTime();
                            }
                        }
                    }
                    var responseStream = response.GetResponseStream();
                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                        responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                        responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);

                    using (var reader = new StreamReader(responseStream, Encoding.Default))
                    {
                        result = reader.ReadToEnd();
                    }
                }
            }
            catch (WebException webError)
            {
                if (webError.Response != null)
                {
                    using (var reader = new StreamReader(webError.Response.GetResponseStream()))
                    {
                        log.Error(reader.ReadToEnd());
                    }
                }
            }
            catch (Exception exc)
            {
                log.Error(exc);
            }
            return result;
        }

        private static HttpWebRequest ConfigureRequest(Uri location, string referer, bool gzipEnabled)
        {
            var request = (HttpWebRequest) WebRequest.Create(location);
            if (!string.IsNullOrEmpty(referer))
                request.Referer = referer;
            request.Accept = @"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            request.UserAgent = GetRandomUserAgent();
            request.Timeout = DEFAULT_DOWNLOAD_TIMEOUT;
            if (gzipEnabled)
                request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");

            return request;
        }

        private static string GetRandomUserAgent()
        {
            var rand = new Random();

            string[] userAgents = {
                                      @"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
                                      ,
                                      @"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)"
                                      ,
                                      @"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.3 (KHTML, like Gecko) Chrome/4.0.223.16 Safari/532.3"
                                      ,
                                      @"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1"
                                  };

            return userAgents[rand.Next(userAgents.Length)];
        }
    }
}